/**
 * 
 */
package edu.umd.clip.lm.util;

import java.util.ArrayList;
import java.util.regex.*;

/**
 * @author Denis Filimonov <den@cs.umd.edu>
 *
 */
public class PTBTokenizer {
	private static final Pattern re1 = Pattern.compile("^(.+)(n't)$");
	private static final Pattern re2 = Pattern.compile("^(.+)('(?:m|s|d|ll|re|ve|))$");
	public static final Pattern split_re = Pattern.compile("\\s+");
	
	public String[] tokenize(String line) {
		line = line.trim();
		String words[] = split_re.split(line);
		return retokenize(words);
	}
	
	public String[] retokenize(String words[]) {
		ArrayList<String> newWords = new ArrayList<String>((int) (1.2 * words.length));
		for(String word : words) {
			Matcher match = re1.matcher(word);
			if (match.matches()) {
				newWords.add(match.group(1));
				newWords.add(match.group(2));
				continue;
			}
			match = re2.matcher(word);
			if (match.matches()) {
				newWords.add(match.group(1));
				newWords.add(match.group(2));
				continue;
			}
			if (word.equalsIgnoreCase("cannot")) {
				newWords.add(word.substring(0, 3));
				newWords.add(word.substring(3));
				continue;
			}
			newWords.add(word);
		}
		return newWords.toArray(new String[newWords.size()]);
	}
}
